#!/usr/bin/env python

import sys, os, lucene, threading, time
from datetime import datetime

"""
This class is loosely based on the Lucene (java implementation) demo class
org.apache.lucene.demo.IndexFiles.

It will take a csv data file as an argument (or use a default source)
and will index data found in this file.

The data is assumed to be in the following format:
id,type,t_name,t_duration,t_a_id,t_a_name,t_num,t_r_id,t_r_name,t_r_attributes,t_r_tracks,t_trm_lookups

The resulting Lucene index will be placed in the current directory and called
'index'.
"""

DEFAULT_SOURCE = "/Users/koch/Projekte/solr3/downloads/mb_tracks.csv"
MAX_DATA = 0

class Ticker(object):

    def __init__(self):
        self.tick = True

    def run(self):
        while self.tick:
            sys.stdout.write('.')
            sys.stdout.flush()
            time.sleep(1.0)

class Indexer(object):
    """Usage: python Indexer <data_file>"""

    def __init__(self, datafile, storeDir, analyzer, verbose=False):
        if not os.path.exists(storeDir):
            os.mkdir(storeDir)
        store = lucene.SimpleFSDirectory(lucene.File(storeDir))
        writer = lucene.IndexWriter(store, analyzer, True,
                                    lucene.IndexWriter.MaxFieldLength.LIMITED)
        writer.setMaxFieldLength(1048576)
        found = self.indexData(datafile, writer, verbose)
        ticker = Ticker()
        print 'optimizing index',
        threading.Thread(target=ticker.run).start()
        writer.optimize()
        writer.close()
        ticker.tick = False
        print 'done indexing %d items' % found

    def indexData(self, fpath, writer, verbose=False):
    	"""parse file and fill index
    	"""
        with open(fpath,'r') as datafile:
            i = 0
            for line in datafile.readlines():
                if i:
                    data = line.split(',')
                    did = data[0]
                    dtyp= data[1]
                    dname = data[2]
                    dartist = data[5]
                    if verbose:
                    	print i,line
                    	print "ID:%s (Typ:%s) '%s' (artist: '%s')" %(
                    		did,dtyp,dname,dartist)                
                    doc = lucene.Document()
                    doc.add(lucene.Field("id", did,
                            lucene.Field.Store.YES,
                            lucene.Field.Index.NOT_ANALYZED))
                    doc.add(lucene.Field("name", dname,
                            lucene.Field.Store.YES,
                            lucene.Field.Index.ANALYZED))
                    doc.add(lucene.Field("artist", dartist,
                            lucene.Field.Store.YES,
                            lucene.Field.Index.ANALYZED))
                    writer.addDocument(doc)
                i+=1
                if MAX_DATA and i>=MAX_DATA:
                    break
                if (i%10000)==0:
                	sys.stdout.write('.')
                	sys.stdout.flush()

        return i        


if __name__ == '__main__':
    if len(sys.argv)==1:
        datafile = DEFAULT_SOURCE
    elif len(sys.argv)==2:
        datafile = sys.argv[1]
    else:
        print IndexData.__doc__
        sys.exit(1)
    if not os.path.exists(datafile):
    	print "data file not found: ", datafile
    	sys.exit(1)
    lucene.initVM()
    print 'using lucene ', lucene.VERSION
    print 'loading data from ', datafile
    start = datetime.now()
    try:
        Indexer(datafile, "index",
                   lucene.StandardAnalyzer(lucene.Version.LUCENE_CURRENT))
        end = datetime.now()
        print "took: %s" % (end - start)
    except Exception, e:
        print "Failed: ", e
